from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

print "Program Begins..."
trecIDANDNumOfPostingsDict = {}
inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sorted_by_XdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument_sortedByTrecID"
inputFileHandler = open(inputFileName1,"r")
lineCounter = 0
currentLine = inputFileHandler.readline()
while currentLine:
    
    if lineCounter % 1000000 == 0:
        print "lineCounter:",lineCounter
    
    currentLineElements = currentLine.strip().split(" ")
    currentTrecID = currentLineElements[0]
    currentNumOfPostingsRecorded = int( currentLineElements[2] )
    currentLine = inputFileHandler.readline()
    if currentTrecID not in trecIDANDNumOfPostingsDict:
        trecIDANDNumOfPostingsDict[currentTrecID] = currentNumOfPostingsRecorded 
    lineCounter += 1
print "len(trecIDANDNumOfPostingsDict):",len(trecIDANDNumOfPostingsDict)
print "trecIDANDNumOfPostingsDict['GX000-00-0002361']:",trecIDANDNumOfPostingsDict['GX000-00-0002361']
inputFileHandler.close()

outputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/indexDocIDANDNumOfPostingsMappingTable"
outputFileHandler = open(outputFileName,"w")


inputFileName2 = "/home/diaosi/web-search-engine-wei/polyIRIndexer/indexTrecIDANDDocIDANDWordSizeInDoc"
inputFileHandler = open(inputFileName2,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTrecID = lineElements[0]
    currentDocIDForIndex = lineElements[1]
    outputFileHandler.write( currentDocIDForIndex + " " + str( trecIDANDNumOfPostingsDict[currentTrecID] ) + "\n")

print "Overall Processing Statistics"
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "outputFileName:",outputFileName
inputFileHandler.close()
outputFileHandler.close()
print "Program Ends."



